package com.yanhui.utils.http;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.yanhui.api.ai.NaturalLanguageApi;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @Description ${END}
 * @Author yanhui
 * @Date 2019-03-10 21:45
 */
public class _httpUtils2 {

    public static void main(String[] args) {
        List<String> resultList = new ArrayList<String>();
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i <10; i++){
            String url = "https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv28&productId=41855078071&score=0&sortType=5&page="+i+"&pageSize=10&isShadowSku=0&fold=1";
            String html = getHtmlResource(url, "GBK");

            JSONObject jsonObject =parseJSONP(html);

            JSONArray comments = jsonObject.getJSONArray("comments");
            for (int j = 0, k=comments.size(); j<k; j++){
                JSONObject jo = comments.getJSONObject(j);
                String content = jo.getString("content");
                //去除非中文
                String rex = "[^\\u4e00-\\u9fa5]";
                Pattern compile = Pattern.compile(rex);
                Matcher matcher = compile.matcher(content);

                content = matcher.replaceAll("");
                sb.append(content);
            }
        }

        //分词
        List<String> a = NaturalLanguageApi.getKeyword(sb.toString());
        for(String str: a){
            System.out.println(str);
        }

    }


    public static JSONObject parseJSONP(String jsoup){
        int startIndex = jsoup.indexOf("(");
        int endIndex = jsoup.lastIndexOf(")");
        String json = jsoup.substring(startIndex+1, endIndex);
        com.alibaba.fastjson.JSONObject jsonObject = JSON.parseObject(json);
        return jsonObject;
    }

    public static String getHtmlResource(String url, String encoding) {
        StringBuilder sb = new StringBuilder();
        InputStreamReader isr = null;
        try {
            URL getUrl = new URL(url);
            URLConnection uc = getUrl.openConnection();
            isr = new InputStreamReader(uc.getInputStream() , encoding);
            BufferedReader br = new BufferedReader(isr);
            String line = "";
            while ((line = br.readLine()) != null){
                sb.append(line);
            }
        } catch (MalformedURLException e) {
            System.out.println("网络链接错误~~~");
            e.printStackTrace();
        } catch (IOException e) {
            System.out.println("网络IO异常~~~");
            e.printStackTrace();
        } finally {
            if(isr!=null){
                try {
                    isr.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return sb.toString();
    }
}
